package com.zx.oschina;

import java.io.Closeable;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.http.HttpEntity;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 获得oschina全部开源组织
 */
public class OpenSourceOrgan {
	
	private static Logger LOGGER = LoggerFactory.getLogger(OpenSourceOrgan.class);
	
	public static void main(String[] args) throws Exception {
		getRoomUrls("https://www.oschina.net/company");
	}

	public static List<String> getRoomUrls(String pageUrl) throws Exception {
		CloseableHttpClient httpclient =  null;
		CloseableHttpResponse response = null;
		
		try {
            httpclient = sslClient();// 设置跳过ssl
			
			HttpGet httpget = new HttpGet(pageUrl); // 创建httpget实例
			
			// 模拟浏览器请求
		    httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
		    httpget.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
		    httpget.setHeader("Accept-Encoding", "gzip, deflate, br");
		    httpget.setHeader("Accept-Language", "zh-CN,zh;q=0.9");
		    httpget.setHeader("Cache-Control", "max-age=0");
		    httpget.setHeader("Connection", "keep-alive");
		    httpget.setHeader("Cookie", "visit-gitee-stars-bts-13-190108=1; aliyungf_tc=AQAAAFygwFtZ5wkAomjGb+PyuIMFRxtF; Hm_lvt_a411c4d1664dd70048ee98afe7b28f0b=1547518505,1547518539,1547518611,1548659983; Hm_lpvt_a411c4d1664dd70048ee98afe7b28f0b=1548662214");
		    httpget.setHeader("Host", "www.oschina.net");
		    httpget.setHeader("Upgrade-Insecure-Requests", "1");
		    
		    response = httpclient.execute(httpget); // 执行get请求
		    
		    LOGGER.info("请求状态", response.getStatusLine().getStatusCode());

		    HttpEntity entity=response.getEntity(); // 获取返回实体
	        
	        String content=EntityUtils.toString(entity, "utf-8");
	        
	        Document doc=Jsoup.parse(content); // 解析网页 得到文档对象
	        
	        List<String> roomUrls = new ArrayList<>();
	        
	        Elements names = doc.select("div .content .header");
	        
	        for (int i = 0; i < names.size(); i++) {
	        	
	        	Element nameEle = names.get(i);
	        	String name = nameEle.html();
	        	String imgUrl = nameEle.parent().parent().select(".image img").attr("src");
	        	
	        	String oschinaUrl = nameEle.parent().parent().attr("href");
	        	
	        	// System.out.println(name + "~~~" + imgUrl + "~~~" + oschinaUrl);
	        	// 生成SQL
	        	System.out.println("INSERT INTO technical_manufacturer(m_name,icon_url,oschina_url) VALUES  ('"+name+"', '"+imgUrl+"', '"+oschinaUrl+"') ;");
	        	
			}

	        
	        return roomUrls;
		} catch (Exception e){
			throw e;
		} finally {
			close(response);
			close(httpclient);
		}
	}
	
	
	/**
	 * @Title: 关闭资源
	 * @param closeable
	 * @date: 2017年12月3日下午9:13:07
	 */
	public static void close(Closeable closeable){
		if(closeable != null)
			try {
				closeable.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
	} 
	
	 /**
     * 在调用SSL之前需要重写验证方法，取消检测SSL
     * 创建ConnectionManager，添加Connection配置信息
     * @return HttpClient 支持https
     */
    private static CloseableHttpClient sslClient() {
        try {
            // 在调用SSL之前需要重写验证方法，取消检测SSL
            X509TrustManager trustManager = new X509TrustManager() {
                @Override public X509Certificate[] getAcceptedIssuers() {
                    return null;
                }
                @Override public void checkClientTrusted(X509Certificate[] xcs, String str) {}
                @Override public void checkServerTrusted(X509Certificate[] xcs, String str) {}
            };
            SSLContext ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
            ctx.init(null, new TrustManager[] { trustManager }, null);
            SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
            // 创建Registry
            RequestConfig requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD_STRICT)
                    .setExpectContinueEnabled(Boolean.TRUE).setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM,AuthSchemes.DIGEST))
                    .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)).build();
            Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                    .register("http", PlainConnectionSocketFactory.INSTANCE)
                    .register("https",socketFactory).build();
            // 创建ConnectionManager，添加Connection配置信息
            PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
            CloseableHttpClient closeableHttpClient = HttpClients.custom().setConnectionManager(connectionManager)
                    .setDefaultRequestConfig(requestConfig).build();
            return closeableHttpClient;
        } catch (KeyManagementException ex) {
            throw new RuntimeException(ex);
        } catch (NoSuchAlgorithmException ex) {
            throw new RuntimeException(ex);
        }
    }
}
